package com.it.bigdata;

import com.it.bigdata.DownLoad.impl.BaiDuTieBaDownLoad;
import com.it.bigdata.domain.TieZi;
import com.it.bigdata.parse.impl.BaiDuTieBaParser;
import com.it.bigdata.schedual.impl.BaiDuTieBaSchedualForQueue;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * 爬虫的主类
 */
public class Splider {
    public static void main(String[]args){
        /**
         *1.下载页面
         *2.解析页面
         *3.存储数据
         */
        //创建下载对象和解析对象
        final BaiDuTieBaDownLoad baiDuTieBaDownLoad = new BaiDuTieBaDownLoad();
        final BaiDuTieBaParser baiDuTieBaParser = new BaiDuTieBaParser();
        final BaiDuTieBaSchedualForQueue baiDuTieBaSchedualForQueue = BaiDuTieBaSchedualForQueue.getBdQueue();



        ExecutorService executorService = Executors.newFixedThreadPool(2);
        //此任务用来下载页面地址
        executorService.execute(new Runnable() {
            @Override
            public void run() {

                while (true){
                    //下载每一页
                    String html = baiDuTieBaDownLoad.download("https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn="+baiDuTieBaSchedualForQueue.getCurrentPageNum());
                    //解析每一页
                    baiDuTieBaParser.parsePageList(html);
                    baiDuTieBaSchedualForQueue.nextPage();
                }
            }
        });
        //此任务用来下载解析具体页面并获取其标题和图片
        executorService.execute(new Runnable() {


            @Override
            public void run() {

                while (true){
                   //从队列(调度器)中获取页面的地址并且从具体的页面中获取标题和图片
                   String pageUrl = baiDuTieBaSchedualForQueue.pop();
                   if (pageUrl!=null){

                       //下载具体的页面并且解析具体的标题和图片
                       //System.out.println(pageUrl);

                       String pageHtml = baiDuTieBaDownLoad.download("https://tieba.baidu.com"+pageUrl);

                       TieZi tieZi = baiDuTieBaParser.parserCurrentToTieZi(pageHtml);

                   }else{
                //       System.out.println("没有可读取的页面");
                   }
               }
            }
        });

    }

}
